{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Understanding Tree SHAP for Simple Models\n",
    "\n",
    "The SHAP value for a feature is the average change in model output by conditioning on that feature when introducing features one at a time over all feature orderings. While this is easy to state, it is challenging to compute. So this notebook is meant to give a few simple examples where we can see how this plays out for very small trees. For arbitrary large trees it is very hard to intuitively guess these values by looking at the tree."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import graphviz\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.tree import DecisionTreeRegressor, export_graphviz\n",
    "\n",
    "import shap"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single split example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 12.2.1 (20241206.2353)\n",
       " -->\n",
       "<!-- Title: Tree Pages: 1 -->\n",
       "<svg width=\"296pt\" height=\"163pt\"\n",
       " viewBox=\"0.00 0.00 296.25 163.25\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 159.25)\">\n",
       "<title>Tree</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-159.25 292.25,-159.25 292.25,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M203,-155.25C203,-155.25 84.25,-155.25 84.25,-155.25 78.25,-155.25 72.25,-149.25 72.25,-143.25 72.25,-143.25 72.25,-103.25 72.25,-103.25 72.25,-97.25 78.25,-91.25 84.25,-91.25 84.25,-91.25 203,-91.25 203,-91.25 209,-91.25 215,-97.25 215,-103.25 215,-103.25 215,-143.25 215,-143.25 215,-149.25 209,-155.25 203,-155.25\"/>\n",
       "<text text-anchor=\"start\" x=\"120\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"126.75\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">0</text>\n",
       "<text text-anchor=\"start\" x=\"133.5\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"80.25\" y=\"-124.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"99\" y=\"-110.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 100</text>\n",
       "<text text-anchor=\"start\" x=\"110.25\" y=\"-96.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.5</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M123.25,-55.25C123.25,-55.25 12,-55.25 12,-55.25 6,-55.25 0,-49.25 0,-43.25 0,-43.25 0,-12 0,-12 0,-6 6,0 12,0 12,0 123.25,0 123.25,0 129.25,0 135.25,-6 135.25,-12 135.25,-12 135.25,-43.25 135.25,-43.25 135.25,-49.25 129.25,-55.25 123.25,-55.25\"/>\n",
       "<text text-anchor=\"start\" x=\"8\" y=\"-37.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"26.75\" y=\"-22.2\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"34.25\" y=\"-6.45\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M118.11,-90.81C111.26,-82.38 103.8,-73.19 96.77,-64.53\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"99.62,-62.48 90.6,-56.92 94.18,-66.89 99.62,-62.48\"/>\n",
       "<text text-anchor=\"middle\" x=\"87.06\" y=\"-75.19\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">True</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M276.25,-55.25C276.25,-55.25 165,-55.25 165,-55.25 159,-55.25 153,-49.25 153,-43.25 153,-43.25 153,-12 153,-12 153,-6 159,0 165,0 165,0 276.25,0 276.25,0 282.25,0 288.25,-6 288.25,-12 288.25,-12 288.25,-43.25 288.25,-43.25 288.25,-49.25 282.25,-55.25 276.25,-55.25\"/>\n",
       "<text text-anchor=\"start\" x=\"161\" y=\"-37.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"179.75\" y=\"-22.2\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"187.25\" y=\"-6.45\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M169.48,-90.81C176.42,-82.38 183.97,-73.19 191.1,-64.53\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"193.71,-66.87 197.36,-56.92 188.3,-62.42 193.71,-66.87\"/>\n",
       "<text text-anchor=\"middle\" x=\"200.74\" y=\"-75.21\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">False</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.sources.Source at 0x1a3bdd5cf50>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# build data\n",
    "N = 100\n",
    "M = 4\n",
    "X = np.zeros((N, M))\n",
    "X.shape\n",
    "y = np.zeros(N)\n",
    "X[: N // 2, 0] = 1\n",
    "y[: N // 2] = 1\n",
    "\n",
    "# fit model\n",
    "single_split_model = DecisionTreeRegressor(max_depth=1)\n",
    "single_split_model.fit(X, y)\n",
    "\n",
    "# draw model\n",
    "dot_data = export_graphviz(\n",
    "    single_split_model,\n",
    "    out_file=None,\n",
    "    filled=True,\n",
    "    rounded=True,\n",
    "    special_characters=True,\n",
    ")\n",
    "graph = graphviz.Source(dot_data)\n",
    "graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explaining the model\n",
    "\n",
    "Note that the bias term is the expected output of the model over the training dataset (0.5). The SHAP value for features not used in the model is always 0, while for $x_0$ it is just the difference between the expected value and the output of the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>x0</th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 0</th>\n",
       "      <th>x</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 1</th>\n",
       "      <th>x</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        x0   x1   x2   x3\n",
       "Example 0 x            1.0  1.0  1.0  1.0\n",
       "          shap_values  0.5  0.0  0.0  0.0\n",
       "Example 1 x            0.0  0.0  0.0  0.0\n",
       "          shap_values -0.5  0.0  0.0  0.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs = [np.ones(M), np.zeros(M)]\n",
    "df = pd.DataFrame()\n",
    "for idx, x in enumerate(xs):\n",
    "    index = pd.MultiIndex.from_product([[f\"Example {idx}\"], [\"x\", \"shap_values\"]])\n",
    "    df = pd.concat(\n",
    "        [\n",
    "            df,\n",
    "            pd.DataFrame(\n",
    "                [x, shap.TreeExplainer(single_split_model).shap_values(x)],\n",
    "                index=index,\n",
    "                columns=[\"x0\", \"x1\", \"x2\", \"x3\"],\n",
    "            ),\n",
    "        ]\n",
    "    )\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two features AND example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use two features in this example. If feature $x_{0} = 1$ AND $x_{1} = 1$, the target value is one, else zero. Hence we call this the AND model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 12.2.1 (20241206.2353)\n",
       " -->\n",
       "<!-- Title: Tree Pages: 1 -->\n",
       "<svg width=\"376pt\" height=\"263pt\"\n",
       " viewBox=\"0.00 0.00 376.25 263.25\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 259.25)\">\n",
       "<title>Tree</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-259.25 372.25,-259.25 372.25,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<path fill=\"#f8e0ce\" stroke=\"black\" d=\"M208.75,-255.25C208.75,-255.25 82.5,-255.25 82.5,-255.25 76.5,-255.25 70.5,-249.25 70.5,-243.25 70.5,-243.25 70.5,-203.25 70.5,-203.25 70.5,-197.25 76.5,-191.25 82.5,-191.25 82.5,-191.25 208.75,-191.25 208.75,-191.25 214.75,-191.25 220.75,-197.25 220.75,-203.25 220.75,-203.25 220.75,-243.25 220.75,-243.25 220.75,-249.25 214.75,-255.25 208.75,-255.25\"/>\n",
       "<text text-anchor=\"start\" x=\"122\" y=\"-238.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"128.75\" y=\"-238.95\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">1</text>\n",
       "<text text-anchor=\"start\" x=\"135.5\" y=\"-238.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"78.5\" y=\"-224.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.188</text>\n",
       "<text text-anchor=\"start\" x=\"101\" y=\"-210.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 100</text>\n",
       "<text text-anchor=\"start\" x=\"108.5\" y=\"-196.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.25</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M123.25,-150.88C123.25,-150.88 12,-150.88 12,-150.88 6,-150.88 0,-144.88 0,-138.88 0,-138.88 0,-107.62 0,-107.62 0,-101.62 6,-95.62 12,-95.62 12,-95.62 123.25,-95.62 123.25,-95.62 129.25,-95.62 135.25,-101.62 135.25,-107.62 135.25,-107.62 135.25,-138.88 135.25,-138.88 135.25,-144.88 129.25,-150.88 123.25,-150.88\"/>\n",
       "<text text-anchor=\"start\" x=\"8\" y=\"-133.57\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"26.75\" y=\"-117.83\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"34.25\" y=\"-102.08\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M120.73,-190.97C112.95,-181.19 104.3,-170.33 96.28,-160.25\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"99.04,-158.1 90.08,-152.46 93.57,-162.46 99.04,-158.1\"/>\n",
       "<text text-anchor=\"middle\" x=\"86.31\" y=\"-170.69\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">True</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M284,-155.25C284,-155.25 165.25,-155.25 165.25,-155.25 159.25,-155.25 153.25,-149.25 153.25,-143.25 153.25,-143.25 153.25,-103.25 153.25,-103.25 153.25,-97.25 159.25,-91.25 165.25,-91.25 165.25,-91.25 284,-91.25 284,-91.25 290,-91.25 296,-97.25 296,-103.25 296,-103.25 296,-143.25 296,-143.25 296,-149.25 290,-155.25 284,-155.25\"/>\n",
       "<text text-anchor=\"start\" x=\"201\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"207.75\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">0</text>\n",
       "<text text-anchor=\"start\" x=\"214.5\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"161.25\" y=\"-124.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"183.75\" y=\"-110.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"191.25\" y=\"-96.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.5</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M170.84,-190.97C177.68,-182.49 185.18,-173.18 192.35,-164.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"194.88,-166.73 198.43,-156.74 189.43,-162.33 194.88,-166.73\"/>\n",
       "<text text-anchor=\"middle\" x=\"202.05\" y=\"-175\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">False</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M203.25,-55.25C203.25,-55.25 92,-55.25 92,-55.25 86,-55.25 80,-49.25 80,-43.25 80,-43.25 80,-12 80,-12 80,-6 86,0 92,0 92,0 203.25,0 203.25,0 209.25,0 215.25,-6 215.25,-12 215.25,-12 215.25,-43.25 215.25,-43.25 215.25,-49.25 209.25,-55.25 203.25,-55.25\"/>\n",
       "<text text-anchor=\"start\" x=\"88\" y=\"-37.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"106.75\" y=\"-22.2\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"114.25\" y=\"-6.45\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M198.77,-90.81C191.83,-82.38 184.28,-73.19 177.15,-64.53\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"179.95,-62.42 170.89,-56.92 174.54,-66.87 179.95,-62.42\"/>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M356.25,-55.25C356.25,-55.25 245,-55.25 245,-55.25 239,-55.25 233,-49.25 233,-43.25 233,-43.25 233,-12 233,-12 233,-6 239,0 245,0 245,0 356.25,0 356.25,0 362.25,0 368.25,-6 368.25,-12 368.25,-12 368.25,-43.25 368.25,-43.25 368.25,-49.25 362.25,-55.25 356.25,-55.25\"/>\n",
       "<text text-anchor=\"start\" x=\"241\" y=\"-37.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"259.75\" y=\"-22.2\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"267.25\" y=\"-6.45\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;4 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>2&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M250.14,-90.81C256.99,-82.38 264.45,-73.19 271.48,-64.53\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"274.07,-66.89 277.65,-56.92 268.63,-62.48 274.07,-66.89\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.sources.Source at 0x1a3bdfa8450>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# build data\n",
    "N = 100\n",
    "M = 4\n",
    "X = np.zeros((N, M))\n",
    "X.shape\n",
    "y = np.zeros(N)\n",
    "X[: 1 * N // 4, 1] = 1\n",
    "X[: N // 2, 0] = 1\n",
    "X[N // 2 : 3 * N // 4, 1] = 1\n",
    "y[: 1 * N // 4] = 1\n",
    "\n",
    "# fit model\n",
    "and_model = DecisionTreeRegressor(max_depth=2)\n",
    "and_model.fit(X, y)\n",
    "\n",
    "# draw model\n",
    "dot_data = export_graphviz(and_model, out_file=None, filled=True, rounded=True, special_characters=True)\n",
    "graph = graphviz.Source(dot_data)\n",
    "graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explaining the model\n",
    "\n",
    "Note that the bias term is the expected output of the model over the training dataset (0.25). The SHAP values for the unused features $x_2$ and $x_3$ are always 0. For $x_0$ and $x_1$ it is just the difference between the expected value (0.25) and the output of the model split equally between them (since they equally contribute to the AND function)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>x0</th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 0</th>\n",
       "      <th>x</th>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>0.375</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 1</th>\n",
       "      <th>x</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>-0.125</td>\n",
       "      <td>-0.125</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          x0     x1   x2   x3\n",
       "Example 0 x            1.000  1.000  1.0  1.0\n",
       "          shap_values  0.375  0.375  0.0  0.0\n",
       "Example 1 x            0.000  0.000  0.0  0.0\n",
       "          shap_values -0.125 -0.125  0.0  0.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs = np.array([np.ones(M), np.zeros(M)])\n",
    "# np.array([np.ones(M), np.zeros(M), np.array([1, 0, 1, 0]), np.array([0, 1, 0, 0])]   # you can also check these examples\n",
    "df = pd.DataFrame()\n",
    "for idx, x in enumerate(xs):\n",
    "    index = pd.MultiIndex.from_product([[f\"Example {idx}\"], [\"x\", \"shap_values\"]])\n",
    "    df = pd.concat(\n",
    "        [\n",
    "            df,\n",
    "            pd.DataFrame(\n",
    "                [x, shap.TreeExplainer(and_model).shap_values(x)],\n",
    "                index=index,\n",
    "                columns=[\"x0\", \"x1\", \"x2\", \"x3\"],\n",
    "            ),\n",
    "        ]\n",
    "    )\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.25)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here is how you get to the Shap values of Example 1: <br>\n",
    "The bias term (`y.mean()`) is 0.25, and the target value is 1. This leaves 1 - 0.25 = 0.75 to split among the relevant features. Since only $x_0$ and $x_1$ contribute to the target value (and to the same extent), it is divided among them, i.e., 0.375 for each."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two features OR example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We do a slight variation of the example above. If $x_{0} = 1$ OR $x_{1} = 1$ the target is 1, else 0. Can you guess the SHAP values without scrolling down?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 12.2.1 (20241206.2353)\n",
       " -->\n",
       "<!-- Title: Tree Pages: 1 -->\n",
       "<svg width=\"376pt\" height=\"263pt\"\n",
       " viewBox=\"0.00 0.00 376.25 263.25\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 259.25)\">\n",
       "<title>Tree</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-259.25 372.25,-259.25 372.25,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<path fill=\"#eca06a\" stroke=\"black\" d=\"M284.75,-255.25C284.75,-255.25 158.5,-255.25 158.5,-255.25 152.5,-255.25 146.5,-249.25 146.5,-243.25 146.5,-243.25 146.5,-203.25 146.5,-203.25 146.5,-197.25 152.5,-191.25 158.5,-191.25 158.5,-191.25 284.75,-191.25 284.75,-191.25 290.75,-191.25 296.75,-197.25 296.75,-203.25 296.75,-203.25 296.75,-243.25 296.75,-243.25 296.75,-249.25 290.75,-255.25 284.75,-255.25\"/>\n",
       "<text text-anchor=\"start\" x=\"198\" y=\"-238.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"204.75\" y=\"-238.95\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">1</text>\n",
       "<text text-anchor=\"start\" x=\"211.5\" y=\"-238.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"154.5\" y=\"-224.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.188</text>\n",
       "<text text-anchor=\"start\" x=\"177\" y=\"-210.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 100</text>\n",
       "<text text-anchor=\"start\" x=\"184.5\" y=\"-196.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.75</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M203,-155.25C203,-155.25 84.25,-155.25 84.25,-155.25 78.25,-155.25 72.25,-149.25 72.25,-143.25 72.25,-143.25 72.25,-103.25 72.25,-103.25 72.25,-97.25 78.25,-91.25 84.25,-91.25 84.25,-91.25 203,-91.25 203,-91.25 209,-91.25 215,-97.25 215,-103.25 215,-103.25 215,-143.25 215,-143.25 215,-149.25 209,-155.25 203,-155.25\"/>\n",
       "<text text-anchor=\"start\" x=\"120\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"126.75\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">0</text>\n",
       "<text text-anchor=\"start\" x=\"133.5\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"80.25\" y=\"-124.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"102.75\" y=\"-110.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"110.25\" y=\"-96.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.5</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M196.73,-190.97C190.05,-182.57 182.72,-173.37 175.71,-164.56\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"178.46,-162.4 169.49,-156.75 172.98,-166.75 178.46,-162.4\"/>\n",
       "<text text-anchor=\"middle\" x=\"165.73\" y=\"-174.98\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">True</text>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M356.25,-150.88C356.25,-150.88 245,-150.88 245,-150.88 239,-150.88 233,-144.88 233,-138.88 233,-138.88 233,-107.62 233,-107.62 233,-101.62 239,-95.62 245,-95.62 245,-95.62 356.25,-95.62 356.25,-95.62 362.25,-95.62 368.25,-101.62 368.25,-107.62 368.25,-107.62 368.25,-138.88 368.25,-138.88 368.25,-144.88 362.25,-150.88 356.25,-150.88\"/>\n",
       "<text text-anchor=\"start\" x=\"241\" y=\"-133.57\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"259.75\" y=\"-117.83\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"267.25\" y=\"-102.08\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;4 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>0&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M246.84,-190.97C254.8,-181.09 263.66,-170.11 271.85,-159.94\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"274.34,-162.43 277.89,-152.45 268.89,-158.04 274.34,-162.43\"/>\n",
       "<text text-anchor=\"middle\" x=\"281.51\" y=\"-170.71\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">False</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M123.25,-55.25C123.25,-55.25 12,-55.25 12,-55.25 6,-55.25 0,-49.25 0,-43.25 0,-43.25 0,-12 0,-12 0,-6 6,0 12,0 12,0 123.25,0 123.25,0 129.25,0 135.25,-6 135.25,-12 135.25,-12 135.25,-43.25 135.25,-43.25 135.25,-49.25 129.25,-55.25 123.25,-55.25\"/>\n",
       "<text text-anchor=\"start\" x=\"8\" y=\"-37.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"26.75\" y=\"-22.2\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"34.25\" y=\"-6.45\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M118.11,-90.81C111.26,-82.38 103.8,-73.19 96.77,-64.53\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"99.62,-62.48 90.6,-56.92 94.18,-66.89 99.62,-62.48\"/>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M276.25,-55.25C276.25,-55.25 165,-55.25 165,-55.25 159,-55.25 153,-49.25 153,-43.25 153,-43.25 153,-12 153,-12 153,-6 159,0 165,0 165,0 276.25,0 276.25,0 282.25,0 288.25,-6 288.25,-12 288.25,-12 288.25,-43.25 288.25,-43.25 288.25,-49.25 282.25,-55.25 276.25,-55.25\"/>\n",
       "<text text-anchor=\"start\" x=\"161\" y=\"-37.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"179.75\" y=\"-22.2\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"187.25\" y=\"-6.45\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M169.48,-90.81C176.42,-82.38 183.97,-73.19 191.1,-64.53\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"193.71,-66.87 197.36,-56.92 188.3,-62.42 193.71,-66.87\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.sources.Source at 0x1a3c11a0f50>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# build data\n",
    "N = 100\n",
    "M = 4\n",
    "X = np.zeros((N, M))\n",
    "X.shape\n",
    "y = np.zeros(N)\n",
    "X[: N // 2, 0] = 1\n",
    "X[: 1 * N // 4, 1] = 1\n",
    "X[N // 2 : 3 * N // 4, 1] = 1\n",
    "y[: N // 2] = 1\n",
    "y[N // 2 : 3 * N // 4] = 1\n",
    "\n",
    "# fit model\n",
    "or_model = DecisionTreeRegressor(max_depth=2)\n",
    "or_model.fit(X, y)\n",
    "\n",
    "# draw model\n",
    "dot_data = export_graphviz(or_model, out_file=None, filled=True, rounded=True, special_characters=True)\n",
    "graph = graphviz.Source(dot_data)\n",
    "graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explaining the model\n",
    "\n",
    "Note that the bias term is the expected output of the model over the training dataset (0.75). The SHAP value for features not used in the model is always 0, while for $x_0$ and $x_1$ it is just the difference between the expected value and the output of the model split equally between them (since they equally contribute to the OR function)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>x0</th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 0</th>\n",
       "      <th>x</th>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>0.125</td>\n",
       "      <td>0.125</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 1</th>\n",
       "      <th>x</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>-0.375</td>\n",
       "      <td>-0.375</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          x0     x1   x2   x3\n",
       "Example 0 x            1.000  1.000  1.0  1.0\n",
       "          shap_values  0.125  0.125  0.0  0.0\n",
       "Example 1 x            0.000  0.000  0.0  0.0\n",
       "          shap_values -0.375 -0.375  0.0  0.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs = np.array([np.ones(M), np.zeros(M)])\n",
    "# np.array([np.ones(M), np.zeros(M), np.array([1, 0, 1, 0]), np.array([0, 1, 0, 0])]   # you can also check these examples\n",
    "df = pd.DataFrame()\n",
    "for idx, x in enumerate(xs):\n",
    "    index = pd.MultiIndex.from_product([[f\"Example {idx}\"], [\"x\", \"shap_values\"]])\n",
    "    df = pd.concat(\n",
    "        [\n",
    "            df,\n",
    "            pd.DataFrame(\n",
    "                [x, shap.TreeExplainer(or_model).shap_values(x)],\n",
    "                index=index,\n",
    "                columns=[\"x0\", \"x1\", \"x2\", \"x3\"],\n",
    "            ),\n",
    "        ]\n",
    "    )\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two feature XOR example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 12.2.1 (20241206.2353)\n",
       " -->\n",
       "<!-- Title: Tree Pages: 1 -->\n",
       "<svg width=\"606pt\" height=\"263pt\"\n",
       " viewBox=\"0.00 0.00 606.25 263.25\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 259.25)\">\n",
       "<title>Tree</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-259.25 602.25,-259.25 602.25,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M358,-255.25C358,-255.25 239.25,-255.25 239.25,-255.25 233.25,-255.25 227.25,-249.25 227.25,-243.25 227.25,-243.25 227.25,-203.25 227.25,-203.25 227.25,-197.25 233.25,-191.25 239.25,-191.25 239.25,-191.25 358,-191.25 358,-191.25 364,-191.25 370,-197.25 370,-203.25 370,-203.25 370,-243.25 370,-243.25 370,-249.25 364,-255.25 358,-255.25\"/>\n",
       "<text text-anchor=\"start\" x=\"275\" y=\"-238.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"281.75\" y=\"-238.95\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">0</text>\n",
       "<text text-anchor=\"start\" x=\"288.5\" y=\"-238.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"235.25\" y=\"-224.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"254\" y=\"-210.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 100</text>\n",
       "<text text-anchor=\"start\" x=\"265.25\" y=\"-196.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.5</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M278,-155.25C278,-155.25 159.25,-155.25 159.25,-155.25 153.25,-155.25 147.25,-149.25 147.25,-143.25 147.25,-143.25 147.25,-103.25 147.25,-103.25 147.25,-97.25 153.25,-91.25 159.25,-91.25 159.25,-91.25 278,-91.25 278,-91.25 284,-91.25 290,-97.25 290,-103.25 290,-103.25 290,-143.25 290,-143.25 290,-149.25 284,-155.25 278,-155.25\"/>\n",
       "<text text-anchor=\"start\" x=\"195\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"201.75\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">1</text>\n",
       "<text text-anchor=\"start\" x=\"208.5\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"155.25\" y=\"-124.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"177.75\" y=\"-110.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"185.25\" y=\"-96.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.5</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M273.09,-190.97C266.16,-182.49 258.57,-173.18 251.3,-164.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"254.18,-162.27 245.15,-156.74 248.76,-166.7 254.18,-162.27\"/>\n",
       "<text text-anchor=\"middle\" x=\"241.67\" y=\"-175.01\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">True</text>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M439,-155.25C439,-155.25 320.25,-155.25 320.25,-155.25 314.25,-155.25 308.25,-149.25 308.25,-143.25 308.25,-143.25 308.25,-103.25 308.25,-103.25 308.25,-97.25 314.25,-91.25 320.25,-91.25 320.25,-91.25 439,-91.25 439,-91.25 445,-91.25 451,-97.25 451,-103.25 451,-103.25 451,-143.25 451,-143.25 451,-149.25 445,-155.25 439,-155.25\"/>\n",
       "<text text-anchor=\"start\" x=\"356\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"362.75\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">1</text>\n",
       "<text text-anchor=\"start\" x=\"369.5\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"316.25\" y=\"-124.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"338.75\" y=\"-110.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"346.25\" y=\"-96.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.5</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;4 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>0&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M324.48,-190.97C331.49,-182.49 339.18,-173.18 346.54,-164.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"349.1,-166.67 352.78,-156.73 343.71,-162.21 349.1,-166.67\"/>\n",
       "<text text-anchor=\"middle\" x=\"356.1\" y=\"-175.03\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">False</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M123.25,-55.25C123.25,-55.25 12,-55.25 12,-55.25 6,-55.25 0,-49.25 0,-43.25 0,-43.25 0,-12 0,-12 0,-6 6,0 12,0 12,0 123.25,0 123.25,0 129.25,0 135.25,-6 135.25,-12 135.25,-12 135.25,-43.25 135.25,-43.25 135.25,-49.25 129.25,-55.25 123.25,-55.25\"/>\n",
       "<text text-anchor=\"start\" x=\"8\" y=\"-37.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"26.75\" y=\"-22.2\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"34.25\" y=\"-6.45\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M167.92,-90.81C152.88,-81.49 136.37,-71.25 121.15,-61.81\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"123,-58.84 112.66,-56.55 119.31,-64.79 123,-58.84\"/>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M276.25,-55.25C276.25,-55.25 165,-55.25 165,-55.25 159,-55.25 153,-49.25 153,-43.25 153,-43.25 153,-12 153,-12 153,-6 159,0 165,0 165,0 276.25,0 276.25,0 282.25,0 288.25,-6 288.25,-12 288.25,-12 288.25,-43.25 288.25,-43.25 288.25,-49.25 282.25,-55.25 276.25,-55.25\"/>\n",
       "<text text-anchor=\"start\" x=\"161\" y=\"-37.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"179.75\" y=\"-22.2\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"187.25\" y=\"-6.45\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M219.3,-90.81C219.46,-83.18 219.64,-74.92 219.81,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"223.3,-67.34 220.01,-57.26 216.3,-67.19 223.3,-67.34\"/>\n",
       "</g>\n",
       "<!-- 5 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>5</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M433.25,-55.25C433.25,-55.25 322,-55.25 322,-55.25 316,-55.25 310,-49.25 310,-43.25 310,-43.25 310,-12 310,-12 310,-6 316,0 322,0 322,0 433.25,0 433.25,0 439.25,0 445.25,-6 445.25,-12 445.25,-12 445.25,-43.25 445.25,-43.25 445.25,-49.25 439.25,-55.25 433.25,-55.25\"/>\n",
       "<text text-anchor=\"start\" x=\"318\" y=\"-37.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"336.75\" y=\"-22.2\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"344.25\" y=\"-6.45\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;5 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>4&#45;&gt;5</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M378.95,-90.81C378.79,-83.18 378.61,-74.92 378.44,-67\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"381.95,-67.19 378.24,-57.26 374.95,-67.34 381.95,-67.19\"/>\n",
       "</g>\n",
       "<!-- 6 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>6</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M586.25,-55.25C586.25,-55.25 475,-55.25 475,-55.25 469,-55.25 463,-49.25 463,-43.25 463,-43.25 463,-12 463,-12 463,-6 469,0 475,0 475,0 586.25,0 586.25,0 592.25,0 598.25,-6 598.25,-12 598.25,-12 598.25,-43.25 598.25,-43.25 598.25,-49.25 592.25,-55.25 586.25,-55.25\"/>\n",
       "<text text-anchor=\"start\" x=\"471\" y=\"-37.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"489.75\" y=\"-22.2\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"497.25\" y=\"-6.45\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;6 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>4&#45;&gt;6</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M430.33,-90.81C445.37,-81.49 461.88,-71.25 477.1,-61.81\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"478.94,-64.79 485.59,-56.55 475.25,-58.84 478.94,-64.79\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.sources.Source at 0x1a3c11a3590>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# build data\n",
    "N = 100\n",
    "M = 4\n",
    "X = np.zeros((N, M))\n",
    "X.shape\n",
    "y = np.zeros(N)\n",
    "X[: N // 2, 0] = 1\n",
    "X[: 1 * N // 4, 1] = 1\n",
    "X[N // 2 : 3 * N // 4, 1] = 1\n",
    "y[1 * N // 4 : N // 2] = 1\n",
    "y[N // 2 : 3 * N // 4] = 1\n",
    "\n",
    "# fit model\n",
    "xor_model = DecisionTreeRegressor(max_depth=2)\n",
    "xor_model.fit(X, y)\n",
    "\n",
    "# draw model\n",
    "dot_data = export_graphviz(xor_model, out_file=None, filled=True, rounded=True, special_characters=True)\n",
    "graph = graphviz.Source(dot_data)\n",
    "graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explaining the model\n",
    "\n",
    "Note that the bias term is the expected output of the model over the training dataset (0.5). The SHAP value for features not used in the model is always 0, while for $x_0$ and $x_1$ it is just the difference between the expected value and the output of the model split equally between them (since they equally contribute to the XOR function)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>x0</th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 0</th>\n",
       "      <th>x</th>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 1</th>\n",
       "      <th>x</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         x0    x1   x2   x3\n",
       "Example 0 x            1.00  1.00  1.0  1.0\n",
       "          shap_values -0.25 -0.25  0.0  0.0\n",
       "Example 1 x            0.00  0.00  0.0  0.0\n",
       "          shap_values -0.25 -0.25  0.0  0.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs = np.array([np.ones(M), np.zeros(M)])\n",
    "# np.array([np.ones(M), np.zeros(M), np.array([1, 0, 1, 0]), np.array([0, 1, 0, 0])]   # you can also check these examples\n",
    "df = pd.DataFrame()\n",
    "for idx, x in enumerate(xs):\n",
    "    index = pd.MultiIndex.from_product([[f\"Example {idx}\"], [\"x\", \"shap_values\"]])\n",
    "    df = pd.concat(\n",
    "        [\n",
    "            df,\n",
    "            pd.DataFrame(\n",
    "                [x, shap.TreeExplainer(xor_model).shap_values(x)],\n",
    "                index=index,\n",
    "                columns=[\"x0\", \"x1\", \"x2\", \"x3\"],\n",
    "            ),\n",
    "        ]\n",
    "    )\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two features AND + feature boost example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 12.2.1 (20241206.2353)\n",
       " -->\n",
       "<!-- Title: Tree Pages: 1 -->\n",
       "<svg width=\"376pt\" height=\"263pt\"\n",
       " viewBox=\"0.00 0.00 376.25 263.25\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 259.25)\">\n",
       "<title>Tree</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-259.25 372.25,-259.25 372.25,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<path fill=\"#f5d0b5\" stroke=\"black\" d=\"M208.75,-255.25C208.75,-255.25 82.5,-255.25 82.5,-255.25 76.5,-255.25 70.5,-249.25 70.5,-243.25 70.5,-243.25 70.5,-203.25 70.5,-203.25 70.5,-197.25 76.5,-191.25 82.5,-191.25 82.5,-191.25 208.75,-191.25 208.75,-191.25 214.75,-191.25 220.75,-197.25 220.75,-203.25 220.75,-203.25 220.75,-243.25 220.75,-243.25 220.75,-249.25 214.75,-255.25 208.75,-255.25\"/>\n",
       "<text text-anchor=\"start\" x=\"122\" y=\"-238.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"128.75\" y=\"-238.95\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">0</text>\n",
       "<text text-anchor=\"start\" x=\"135.5\" y=\"-238.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"78.5\" y=\"-224.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.688</text>\n",
       "<text text-anchor=\"start\" x=\"101\" y=\"-210.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 100</text>\n",
       "<text text-anchor=\"start\" x=\"108.5\" y=\"-196.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.75</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1</title>\n",
       "<path fill=\"#ffffff\" stroke=\"black\" d=\"M123.25,-150.88C123.25,-150.88 12,-150.88 12,-150.88 6,-150.88 0,-144.88 0,-138.88 0,-138.88 0,-107.62 0,-107.62 0,-101.62 6,-95.62 12,-95.62 12,-95.62 123.25,-95.62 123.25,-95.62 129.25,-95.62 135.25,-101.62 135.25,-107.62 135.25,-107.62 135.25,-138.88 135.25,-138.88 135.25,-144.88 129.25,-150.88 123.25,-150.88\"/>\n",
       "<text text-anchor=\"start\" x=\"8\" y=\"-133.57\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"26.75\" y=\"-117.83\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"34.25\" y=\"-102.08\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 0.0</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M120.73,-190.97C112.95,-181.19 104.3,-170.33 96.28,-160.25\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"99.04,-158.1 90.08,-152.46 93.57,-162.46 99.04,-158.1\"/>\n",
       "<text text-anchor=\"middle\" x=\"86.31\" y=\"-170.69\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">True</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>2</title>\n",
       "<path fill=\"#eca06a\" stroke=\"black\" d=\"M284,-155.25C284,-155.25 165.25,-155.25 165.25,-155.25 159.25,-155.25 153.25,-149.25 153.25,-143.25 153.25,-143.25 153.25,-103.25 153.25,-103.25 153.25,-97.25 159.25,-91.25 165.25,-91.25 165.25,-91.25 284,-91.25 284,-91.25 290,-91.25 296,-97.25 296,-103.25 296,-103.25 296,-143.25 296,-143.25 296,-149.25 290,-155.25 284,-155.25\"/>\n",
       "<text text-anchor=\"start\" x=\"201\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">x</text>\n",
       "<text text-anchor=\"start\" x=\"207.75\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" baseline-shift=\"sub\" font-size=\"14.00\">1</text>\n",
       "<text text-anchor=\"start\" x=\"214.5\" y=\"-138.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\"> ≤ 0.5</text>\n",
       "<text text-anchor=\"start\" x=\"161.25\" y=\"-124.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.25</text>\n",
       "<text text-anchor=\"start\" x=\"183.75\" y=\"-110.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 50</text>\n",
       "<text text-anchor=\"start\" x=\"191.25\" y=\"-96.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.5</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M170.84,-190.97C177.68,-182.49 185.18,-173.18 192.35,-164.28\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"194.88,-166.73 198.43,-156.74 189.43,-162.33 194.88,-166.73\"/>\n",
       "<text text-anchor=\"middle\" x=\"202.05\" y=\"-175\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">False</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<path fill=\"#f2c09c\" stroke=\"black\" d=\"M203.25,-55.25C203.25,-55.25 92,-55.25 92,-55.25 86,-55.25 80,-49.25 80,-43.25 80,-43.25 80,-12 80,-12 80,-6 86,0 92,0 92,0 203.25,0 203.25,0 209.25,0 215.25,-6 215.25,-12 215.25,-12 215.25,-43.25 215.25,-43.25 215.25,-49.25 209.25,-55.25 203.25,-55.25\"/>\n",
       "<text text-anchor=\"start\" x=\"88\" y=\"-37.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"106.75\" y=\"-22.2\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"114.25\" y=\"-6.45\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 1.0</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M198.77,-90.81C191.83,-82.38 184.28,-73.19 177.15,-64.53\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"179.95,-62.42 170.89,-56.92 174.54,-66.87 179.95,-62.42\"/>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4</title>\n",
       "<path fill=\"#e58139\" stroke=\"black\" d=\"M356.25,-55.25C356.25,-55.25 245,-55.25 245,-55.25 239,-55.25 233,-49.25 233,-43.25 233,-43.25 233,-12 233,-12 233,-6 239,0 245,0 245,0 356.25,0 356.25,0 362.25,0 368.25,-6 368.25,-12 368.25,-12 368.25,-43.25 368.25,-43.25 368.25,-49.25 362.25,-55.25 356.25,-55.25\"/>\n",
       "<text text-anchor=\"start\" x=\"241\" y=\"-37.95\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">squared_error = 0.0</text>\n",
       "<text text-anchor=\"start\" x=\"259.75\" y=\"-22.2\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">samples = 25</text>\n",
       "<text text-anchor=\"start\" x=\"267.25\" y=\"-6.45\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">value = 2.0</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;4 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>2&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M250.14,-90.81C256.99,-82.38 264.45,-73.19 271.48,-64.53\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"274.07,-66.89 277.65,-56.92 268.63,-62.48 274.07,-66.89\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.sources.Source at 0x1a3c117a650>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# build data\n",
    "N = 100\n",
    "M = 4\n",
    "X = np.zeros((N, M))\n",
    "X.shape\n",
    "y = np.zeros(N)\n",
    "X[: N // 2, 0] = 1\n",
    "X[: 1 * N // 4, 1] = 1\n",
    "X[N // 2 : 3 * N // 4, 1] = 1\n",
    "y[: 1 * N // 4] = 1\n",
    "y[: N // 2] += 1\n",
    "\n",
    "# fit model\n",
    "and_fb_model = DecisionTreeRegressor(max_depth=2)\n",
    "and_fb_model.fit(X, y)\n",
    "\n",
    "# draw model\n",
    "dot_data = export_graphviz(and_fb_model, out_file=None, filled=True, rounded=True, special_characters=True)\n",
    "graph = graphviz.Source(dot_data)\n",
    "graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explain the model\n",
    "\n",
    "Note that the bias term is the expected output of the model over the training dataset (0.75). The SHAP value for features not used in the model is always 0, while for $x_0$ and $x_1$ it is just the difference between the expected value and the output of the model split equally between them (since they equally contribute to the AND function), plus an extra 0.5 impact for $x_0$ since it has an effect of $1.0$ all by itself (+0.5 if it is on and -0.5 if it is off)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>x0</th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 0</th>\n",
       "      <th>x</th>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>0.875</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Example 1</th>\n",
       "      <th>x</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shap_values</th>\n",
       "      <td>-0.625</td>\n",
       "      <td>-0.125</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          x0     x1   x2   x3\n",
       "Example 0 x            1.000  1.000  1.0  1.0\n",
       "          shap_values  0.875  0.375  0.0  0.0\n",
       "Example 1 x            0.000  0.000  0.0  0.0\n",
       "          shap_values -0.625 -0.125  0.0  0.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs = np.array([np.ones(M), np.zeros(M)])\n",
    "# np.array([np.ones(M), np.zeros(M), np.array([1, 0, 1, 0]), np.array([0, 1, 0, 0])]   # you can also check these examples\n",
    "df = pd.DataFrame()\n",
    "for idx, x in enumerate(xs):\n",
    "    index = pd.MultiIndex.from_product([[f\"Example {idx}\"], [\"x\", \"shap_values\"]])\n",
    "    df = pd.concat(\n",
    "        [\n",
    "            df,\n",
    "            pd.DataFrame(\n",
    "                [x, shap.TreeExplainer(and_fb_model).shap_values(x)],\n",
    "                index=index,\n",
    "                columns=[\"x0\", \"x1\", \"x2\", \"x3\"],\n",
    "            ),\n",
    "        ]\n",
    "    )\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
